{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![title](img/text_cnn.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = 5000\n",
    "MAX_LEN = 400\n",
    "BATCH_SIZE = 32\n",
    "EMBED_DIM = 50\n",
    "FILTERS = 250\n",
    "N_CLASS = 2\n",
    "N_EPOCH = 2\n",
    "LR = {'start': 5e-3, 'end': 5e-4, 'steps': 1500}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(x, mode):\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    \n",
    "    x = tf.contrib.layers.embed_sequence(x, VOCAB_SIZE, EMBED_DIM)\n",
    "    x = tf.layers.dropout(x, 0.2, training=is_training)\n",
    "    \n",
    "    feat_map = []\n",
    "    for k_size in [3, 4, 5]:\n",
    "        _x = tf.layers.conv1d(x, FILTERS, k_size, activation=tf.nn.relu)\n",
    "        _x = tf.layers.max_pooling1d(_x, _x.get_shape().as_list()[1], 1)\n",
    "        _x = tf.reshape(_x, (tf.shape(x)[0], FILTERS))\n",
    "        feat_map.append(_x)\n",
    "    x = tf.concat(feat_map, -1)\n",
    "    \n",
    "    x = tf.layers.dropout(x, 0.2, training=is_training)\n",
    "    x = tf.layers.dense(x, FILTERS, tf.nn.relu)\n",
    "    logits = tf.layers.dense(x, N_CLASS)\n",
    "    return logits\n",
    "\n",
    "\n",
    "def model_fn(features, labels, mode):\n",
    "    logits = forward(features, mode)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        preds = tf.argmax(logits, -1)\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions=preds)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        global_step = tf.train.get_global_step()\n",
    "\n",
    "        lr_op = tf.train.exponential_decay(\n",
    "            LR['start'], global_step, LR['steps'], LR['end']/LR['start'])\n",
    "\n",
    "        loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "            logits=logits, labels=labels))\n",
    "\n",
    "        train_op = tf.train.AdamOptimizer(lr_op).minimize(\n",
    "            loss_op, global_step=global_step)\n",
    "\n",
    "        lth = tf.train.LoggingTensorHook({'lr': lr_op}, every_n_iter=100)\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[lth])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x11318b748>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.69989, step = 1\n",
      "INFO:tensorflow:lr = 0.005\n",
      "INFO:tensorflow:global_step/sec: 7.0603\n",
      "INFO:tensorflow:loss = 0.43512595, step = 101 (14.165 sec)\n",
      "INFO:tensorflow:lr = 0.004288479 (14.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.94101\n",
      "INFO:tensorflow:loss = 0.3178803, step = 201 (14.407 sec)\n",
      "INFO:tensorflow:lr = 0.0036782112 (14.407 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.54246\n",
      "INFO:tensorflow:loss = 0.50463146, step = 301 (15.285 sec)\n",
      "INFO:tensorflow:lr = 0.0031547868 (15.285 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.77713\n",
      "INFO:tensorflow:loss = 0.29612482, step = 401 (14.756 sec)\n",
      "INFO:tensorflow:lr = 0.0027058476 (14.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.93793\n",
      "INFO:tensorflow:loss = 0.12608042, step = 501 (14.413 sec)\n",
      "INFO:tensorflow:lr = 0.0023207942 (14.413 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.27089\n",
      "INFO:tensorflow:loss = 0.28015432, step = 601 (13.753 sec)\n",
      "INFO:tensorflow:lr = 0.0019905358 (13.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.90494\n",
      "INFO:tensorflow:loss = 0.1181194, step = 701 (14.483 sec)\n",
      "INFO:tensorflow:lr = 0.0017072745 (14.483 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.15385376.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Validation Accuracy: 0.8926\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 783 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.27178204, step = 783\n",
      "INFO:tensorflow:lr = 0.0015053472\n",
      "INFO:tensorflow:global_step/sec: 7.56322\n",
      "INFO:tensorflow:loss = 0.27009153, step = 883 (13.223 sec)\n",
      "INFO:tensorflow:lr = 0.00129113 (13.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.33093\n",
      "INFO:tensorflow:loss = 0.08282128, step = 983 (13.641 sec)\n",
      "INFO:tensorflow:lr = 0.001107397 (13.641 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.29224\n",
      "INFO:tensorflow:loss = 0.2913481, step = 1083 (13.713 sec)\n",
      "INFO:tensorflow:lr = 0.00094980985 (13.713 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.83338\n",
      "INFO:tensorflow:loss = 0.10225633, step = 1183 (12.766 sec)\n",
      "INFO:tensorflow:lr = 0.00081464805 (12.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.17165\n",
      "INFO:tensorflow:loss = 0.08282864, step = 1283 (13.944 sec)\n",
      "INFO:tensorflow:lr = 0.0006987203 (13.945 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.30244\n",
      "INFO:tensorflow:loss = 0.13309486, step = 1383 (13.694 sec)\n",
      "INFO:tensorflow:lr = 0.00059928955 (13.693 sec)\n",
      "INFO:tensorflow:global_step/sec: 7.45642\n",
      "INFO:tensorflow:loss = 0.26792693, step = 1483 (13.411 sec)\n",
      "INFO:tensorflow:lr = 0.00051400816 (13.411 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1564 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.07324343.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmph49ue1wi/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Validation Accuracy: 0.8961\n",
      "\n"
     ]
    }
   ],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.imdb.load_data(num_words=VOCAB_SIZE)\n",
    "X_train = tf.keras.preprocessing.sequence.pad_sequences(X_train, MAX_LEN)\n",
    "X_test = tf.keras.preprocessing.sequence.pad_sequences(X_test, MAX_LEN)\n",
    "\n",
    "estimator = tf.estimator.Estimator(model_fn)\n",
    "\n",
    "for _ in range(N_EPOCH):\n",
    "    estimator.train(tf.estimator.inputs.numpy_input_fn(\n",
    "        x = X_train, y = y_train,\n",
    "        batch_size = BATCH_SIZE,\n",
    "        shuffle = True))\n",
    "    y_pred = np.fromiter(estimator.predict(tf.estimator.inputs.numpy_input_fn(\n",
    "        x = X_test,\n",
    "        batch_size = BATCH_SIZE,\n",
    "        shuffle = False)), np.int32)\n",
    "    print(\"\\nValidation Accuracy: %.4f\\n\" % (y_pred==y_test).mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
